package priv.lirenhe.dissertation.component.data_parser.url_parser;

import java.util.ArrayList;
import java.util.List;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import priv.lirenhe.dissertation.component.data_parser.util.UrlUtil;

public class WebContentToUrlParser {
	private String webContent;
	
	public WebContentToUrlParser(String webContent){
		this.webContent = webContent;
	}
	
	public List<String> getUrlOfList(String mainUrl){
		
		List<String> listOfUrls = new ArrayList<String>();
		if(this.webContent != null){
			try{
				Document document = Jsoup.parse(this.webContent);
				Elements aTags = document.select("a");
				String href = null;
				if(aTags != null && !aTags.isEmpty()){
					for(Element aTag:aTags){
						href = aTag.attr("href");
						if(href != null){
							href = UrlUtil.resolveUrl(href);
						}else{
							continue;
						}
						
						if(!UrlUtil.isUrlFromUrlHead(href)){
							if(UrlUtil.isUrlFromUrlHead(mainUrl)){
								mainUrl = UrlUtil.resolveMainUrl(mainUrl);
								if(mainUrl.endsWith("/")){
									if(href.startsWith("/")){
										href = mainUrl + href.substring(1,href.length());
									}else{
										href = mainUrl + "/" + href;
									}
								}else{
									if(href.startsWith("/")){
										href = mainUrl + href;
									}else{
										href = mainUrl + "/" + href;
									}
								}
							}
						}
						if(href.contains("javascript:")){
							continue;
						}
						
						listOfUrls.add(href);
					}
				}
			}catch(Exception e){
				e.printStackTrace();
			}
			
		}
		
		return listOfUrls;
	}
	
}
